package com.chance.cc.crawler.development.bootstrap.guoke;

import com.chance.cc.crawler.core.CrawlerEnum;
import com.chance.cc.crawler.core.filter.FilterUtils;
import com.chance.cc.crawler.core.record.CrawlerRequestRecord;
import com.chance.cc.crawler.development.controller.DevCrawlerController;
import org.apache.commons.codec.digest.DigestUtils;
import org.junit.Test;

public class GuoKeStarter {
    private static final String DOMAIN = "guokr";
    private static final String SITE = "news";
    private static final String START_URL = "https://www.guokr.com/";

    public static void main(String[] args) {
        CrawlerRequestRecord crawlerRequestRecord = CrawlerRequestRecord.builder()
                .startPageRequest(DOMAIN, CrawlerEnum.CrawlerRequestType.turnPage)
                .domain(DOMAIN)
                .httpUrl(START_URL)
                .recordKey(START_URL)
                .releaseTime(System.currentTimeMillis())
                .filter(CrawlerEnum.CrawlerRecordFilter.keyOrDateRange)
                .addFilterInfo(FilterUtils.memoryFilterKeyInfo(DOMAIN))
                .addFilterInfo(FilterUtils.dateRangeFilterInfo(24*7,null))
                .resultLabelTag(CrawlerEnum.CrawlerDataType.article)
                .resultLabelTag(CrawlerEnum.CrawlerDataType.interaction)
                .build();
        crawlerRequestRecord.tagsCreator().bizTags().addDomain(DOMAIN);
        crawlerRequestRecord.tagsCreator().bizTags().addSite(SITE);

        DevCrawlerController.builder()
                .requestRecord(crawlerRequestRecord)
                .triggerInfo(DOMAIN,DOMAIN,System.currentTimeMillis(),DOMAIN)
                .crawlerRequestQueue(DevCrawlerController.devRequestQueue(DOMAIN))
                .consoleResultPipeline()
                .fileResultPipeline("F:\\changxi\\web_data\\guokr.log",false)
                .build("com.chance.cc.crawler.development.scripts.guoke.GuoKrScript")
                .start();
    }
}
